package googlescrapper;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.swing.JFileChooser;
import javax.swing.JTextField;
import javax.swing.SwingUtilities;
import javax.swing.table.DefaultTableModel;
import javax.swing.text.BadLocationException;
import javax.swing.text.StyledDocument;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * Google keyword scrapper with proxy
 * @author Shaiful Islam (palash)
 * @email kuvic16@gmail.com
 */
public class GoogleScrapper extends javax.swing.JFrame implements Notifier{

    /**
     * Creates new form Test
     */
    public GoogleScrapper() {
        initComponents();        
        tblProxyList.setModel(new javax.swing.table.DefaultTableModel(rows, IConstant.headers));
        //tblOutput.setModel(new javax.swing.table.DefaultTableModel(null, IConstant.outputHeaders));
        model = new DefaultTableModel(null, IConstant.outputHeaders);
        tblOutput.setModel(model);
    }

    /**
     * This method is called from within the constructor to initialize the form.
     * WARNING: Do NOT modify this code. The content of this method is always
     * regenerated by the Form Editor.
     */
    @SuppressWarnings("unchecked")
    // <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
    private void initComponents() {

        buttonGroup1 = new javax.swing.ButtonGroup();
        buttonGroup2 = new javax.swing.ButtonGroup();
        jTabbedPane2 = new javax.swing.JTabbedPane();
        jPanel1 = new javax.swing.JPanel();
        lblSearchTerms = new javax.swing.JLabel();
        txtKeywordFile = new javax.swing.JTextField();
        btnKeywordFile = new javax.swing.JButton();
        btnSearch = new javax.swing.JButton();
        jScrollPane3 = new javax.swing.JScrollPane();
        consle = new javax.swing.JTextPane();
        jScrollPane2 = new javax.swing.JScrollPane();
        tblOutput = new javax.swing.JTable();
        btnDownloadCSVFile = new javax.swing.JButton();
        jLabel3 = new javax.swing.JLabel();
        jLabel4 = new javax.swing.JLabel();
        rdHttpProxy = new javax.swing.JRadioButton();
        txtKeyword = new javax.swing.JTextField();
        jLabel5 = new javax.swing.JLabel();
        rdHttpsProxy = new javax.swing.JRadioButton();
        rdMTOnlyGooogle = new javax.swing.JRadioButton();
        rdMTAll = new javax.swing.JRadioButton();
        jPanel2 = new javax.swing.JPanel();
        lblSearchTerms1 = new javax.swing.JLabel();
        jLabel1 = new javax.swing.JLabel();
        btnProxyLoad = new javax.swing.JButton();
        jScrollPane1 = new javax.swing.JScrollPane();
        tblProxyList = new javax.swing.JTable();
        jLabel2 = new javax.swing.JLabel();
        lblTotal = new javax.swing.JLabel();

        setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);
        setTitle("Google Scrapper for keywords");

        lblSearchTerms.setFont(new java.awt.Font("Tahoma", 1, 13)); // NOI18N
        lblSearchTerms.setText("Google Keyword Search Terms Input File");

        txtKeywordFile.setEditable(false);

        btnKeywordFile.setText("Select Keyword File");
        btnKeywordFile.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                btnKeywordFileActionPerformed(evt);
            }
        });

        btnSearch.setBackground(new java.awt.Color(0, 153, 153));
        btnSearch.setText("Search");
        btnSearch.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                btnSearchActionPerformed(evt);
            }
        });

        consle.setEditable(false);
        consle.setBackground(new java.awt.Color(204, 204, 255));
        jScrollPane3.setViewportView(consle);

        tblOutput.setModel(new javax.swing.table.DefaultTableModel(
            new Object [][] {
                {null, null, null, null},
                {null, null, null, null},
                {null, null, null, null},
                {null, null, null, null}
            },
            new String [] {
                "Title 1", "Title 2", "Title 3", "Title 4"
            }
        ));
        jScrollPane2.setViewportView(tblOutput);

        btnDownloadCSVFile.setBackground(new java.awt.Color(204, 204, 255));
        btnDownloadCSVFile.setText("Download CSV File");
        btnDownloadCSVFile.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                btnDownloadCSVFileActionPerformed(evt);
            }
        });

        jLabel3.setText("Output");

        jLabel4.setText("Log");

        buttonGroup1.add(rdHttpProxy);
        rdHttpProxy.setSelected(true);
        rdHttpProxy.setText("HTTPProxy");
        rdHttpProxy.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                rdHttpProxyActionPerformed(evt);
            }
        });

        jLabel5.setText("Keyword");

        buttonGroup1.add(rdHttpsProxy);
        rdHttpsProxy.setText("HTTPSProxy");

        buttonGroup2.add(rdMTOnlyGooogle);
        rdMTOnlyGooogle.setText("Multithreding for only Google Searching ");

        buttonGroup2.add(rdMTAll);
        rdMTAll.setSelected(true);
        rdMTAll.setText("Multithreding for all Searching ");
        rdMTAll.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                rdMTAllActionPerformed(evt);
            }
        });

        javax.swing.GroupLayout jPanel1Layout = new javax.swing.GroupLayout(jPanel1);
        jPanel1.setLayout(jPanel1Layout);
        jPanel1Layout.setHorizontalGroup(
            jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(jPanel1Layout.createSequentialGroup()
                .addContainerGap()
                .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addComponent(lblSearchTerms, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                    .addGroup(jPanel1Layout.createSequentialGroup()
                        .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                            .addComponent(jScrollPane2)
                            .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, jPanel1Layout.createSequentialGroup()
                                .addGap(0, 0, Short.MAX_VALUE)
                                .addComponent(btnDownloadCSVFile))
                            .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, jPanel1Layout.createSequentialGroup()
                                .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                                    .addComponent(jScrollPane3)
                                    .addGroup(jPanel1Layout.createSequentialGroup()
                                        .addComponent(txtKeywordFile, javax.swing.GroupLayout.PREFERRED_SIZE, 574, javax.swing.GroupLayout.PREFERRED_SIZE)
                                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                                        .addComponent(btnKeywordFile)))
                                .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                                    .addGroup(jPanel1Layout.createSequentialGroup()
                                        .addGap(26, 26, 26)
                                        .addComponent(jLabel5)
                                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                                        .addComponent(txtKeyword, javax.swing.GroupLayout.PREFERRED_SIZE, 34, javax.swing.GroupLayout.PREFERRED_SIZE)
                                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                                        .addComponent(btnSearch, javax.swing.GroupLayout.PREFERRED_SIZE, 89, javax.swing.GroupLayout.PREFERRED_SIZE))
                                    .addGroup(jPanel1Layout.createSequentialGroup()
                                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
                                        .addComponent(rdHttpProxy)
                                        .addGap(38, 38, 38)
                                        .addComponent(rdHttpsProxy))
                                    .addGroup(jPanel1Layout.createSequentialGroup()
                                        .addGap(6, 6, 6)
                                        .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                                            .addComponent(rdMTAll)
                                            .addComponent(rdMTOnlyGooogle))))
                                .addGap(20, 20, 20)))
                        .addContainerGap())
                    .addGroup(jPanel1Layout.createSequentialGroup()
                        .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                            .addComponent(jLabel4, javax.swing.GroupLayout.PREFERRED_SIZE, 27, javax.swing.GroupLayout.PREFERRED_SIZE)
                            .addComponent(jLabel3))
                        .addGap(0, 930, Short.MAX_VALUE))))
        );
        jPanel1Layout.setVerticalGroup(
            jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(jPanel1Layout.createSequentialGroup()
                .addContainerGap()
                .addComponent(lblSearchTerms)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
                    .addComponent(txtKeyword, javax.swing.GroupLayout.Alignment.TRAILING, javax.swing.GroupLayout.PREFERRED_SIZE, 29, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                        .addComponent(txtKeywordFile, javax.swing.GroupLayout.PREFERRED_SIZE, 35, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addComponent(btnKeywordFile, javax.swing.GroupLayout.PREFERRED_SIZE, 35, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addComponent(btnSearch, javax.swing.GroupLayout.PREFERRED_SIZE, 35, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addComponent(jLabel5)))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addComponent(jLabel4)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false)
                    .addGroup(jPanel1Layout.createSequentialGroup()
                        .addComponent(jScrollPane3, javax.swing.GroupLayout.PREFERRED_SIZE, 183, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(jLabel3)
                        .addGap(4, 4, 4))
                    .addGroup(jPanel1Layout.createSequentialGroup()
                        .addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                            .addComponent(rdHttpProxy)
                            .addComponent(rdHttpsProxy))
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
                        .addComponent(rdMTOnlyGooogle)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(rdMTAll)
                        .addGap(56, 56, 56)))
                .addComponent(jScrollPane2, javax.swing.GroupLayout.PREFERRED_SIZE, 152, javax.swing.GroupLayout.PREFERRED_SIZE)
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                .addComponent(btnDownloadCSVFile, javax.swing.GroupLayout.PREFERRED_SIZE, 35, javax.swing.GroupLayout.PREFERRED_SIZE)
                .addContainerGap())
        );

        jTabbedPane2.addTab("Search", jPanel1);

        lblSearchTerms1.setFont(new java.awt.Font("Tahoma", 1, 13)); // NOI18N
        lblSearchTerms1.setText("Google proxy listed below");

        jLabel1.setText("(from http://www.google-proxy.net)");

        btnProxyLoad.setText("Reload Proxy");
        btnProxyLoad.setActionCommand("Re-load Proxy");
        btnProxyLoad.addActionListener(new java.awt.event.ActionListener() {
            public void actionPerformed(java.awt.event.ActionEvent evt) {
                btnProxyLoadActionPerformed(evt);
            }
        });

        tblProxyList.setModel(new javax.swing.table.DefaultTableModel(
            new Object [][] {
                {null, null, null, null},
                {null, null, null, null},
                {null, null, null, null},
                {null, null, null, null}
            },
            new String [] {
                "Title 1", "Title 2", "Title 3", "Title 4"
            }
        ));
        jScrollPane1.setViewportView(tblProxyList);

        jLabel2.setText("Total:");

        lblTotal.setFont(new java.awt.Font("Tahoma", 0, 14)); // NOI18N
        lblTotal.setText("0");

        javax.swing.GroupLayout jPanel2Layout = new javax.swing.GroupLayout(jPanel2);
        jPanel2.setLayout(jPanel2Layout);
        jPanel2Layout.setHorizontalGroup(
            jPanel2Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(javax.swing.GroupLayout.Alignment.TRAILING, jPanel2Layout.createSequentialGroup()
                .addContainerGap()
                .addGroup(jPanel2Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING)
                    .addComponent(jScrollPane1)
                    .addGroup(jPanel2Layout.createSequentialGroup()
                        .addComponent(lblSearchTerms1)
                        .addGap(15, 15, 15)
                        .addComponent(jLabel1)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED, 428, Short.MAX_VALUE)
                        .addComponent(jLabel2)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(lblTotal, javax.swing.GroupLayout.PREFERRED_SIZE, 34, javax.swing.GroupLayout.PREFERRED_SIZE)
                        .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
                        .addComponent(btnProxyLoad)))
                .addContainerGap())
        );
        jPanel2Layout.setVerticalGroup(
            jPanel2Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addGroup(jPanel2Layout.createSequentialGroup()
                .addContainerGap()
                .addGroup(jPanel2Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
                    .addComponent(lblSearchTerms1)
                    .addComponent(jLabel1)
                    .addComponent(btnProxyLoad, javax.swing.GroupLayout.PREFERRED_SIZE, 35, javax.swing.GroupLayout.PREFERRED_SIZE)
                    .addComponent(jLabel2)
                    .addComponent(lblTotal))
                .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
                .addComponent(jScrollPane1, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
                .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
        );

        jTabbedPane2.addTab("Proxy", jPanel2);

        javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane());
        getContentPane().setLayout(layout);
        layout.setHorizontalGroup(
            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addComponent(jTabbedPane2)
        );
        layout.setVerticalGroup(
            layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
            .addComponent(jTabbedPane2)
        );

        jTabbedPane2.getAccessibleContext().setAccessibleName("Search");
        jTabbedPane2.getAccessibleContext().setAccessibleDescription("");

        pack();
    }// </editor-fold>//GEN-END:initComponents
    
    //ROOT_DOMAIN, ACTUAL_DESTINATION, GOOGLE_ADVERTISING_FLAG
        
    private Object[][] rows = null;
    //private Object[][] outputRows = null;
    private List<String> keyword = null;
    private List<Output> outputList = new ArrayList<Output>();
    private DefaultTableModel model;
    public String proxy;
    public String threadSettings;
    public Map<String, List<Output>> mList = new HashMap<String, List<Output>>();
            
    private void btnKeywordFileActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_btnKeywordFileActionPerformed
        if (selectSearchTermFile())
        {
            //readSearchTermFile();       
        }
    }//GEN-LAST:event_btnKeywordFileActionPerformed

    private void btnProxyLoadActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_btnProxyLoadActionPerformed
        loadProxy(IConstant.PROXY_URL);
    }//GEN-LAST:event_btnProxyLoadActionPerformed

    private void btnSearchActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_btnSearchActionPerformed
        
//        Set<Thread> threadSet = Thread.getAllStackTraces().keySet();
//        Thread[] threadArray = threadSet.toArray(new Thread[threadSet.size()]);
//        for(Thread t : threadArray){
//            System.out.println(t.getName());
//        }
        
        // keyword select
        if(!(keyword != null && keyword.size()>0)){
            logIt("Keyword file not selected. Please select the keyword file!\n");
            return;
        }
        
        // proxy loading
        if (!(rows != null && rows.length > 0)) {
            logIt("No proxy avialable\n");
            logIt("Searching proxy...\n");
            loadProxy(IConstant.PROXY_URL);
            logIt(rows.length + " proxy found\n");
        }
        
        this.setProxy("HTTPS");
        if(rdHttpProxy.isSelected()){
            this.setProxy("HTTP");
        }
        
        this.setThreadSettings("ALL");
        if(rdMTOnlyGooogle.isSelected()){
            this.setThreadSettings("");
        }
        
        // read keyword size
        int keywordsize = 0;
        if(txtKeyword.getText() != null && txtKeyword.getText().length()>0){
            keywordsize = Integer.valueOf(txtKeyword.getText());
        }
        
        // configure looging
        java.util.logging.Logger.getLogger("com.gargoylesoftware").setLevel(Level.OFF);
        System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
        LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
        java.util.logging.Logger.getLogger("org.apache.commons.httpclient").setLevel(Level.OFF);
        
        for(int i=0; i<keywordsize; i++){
            GoogleSearching g1 = new GoogleSearching(keyword.get(i)+ 1 , keyword.get(i), consle, rows, outputList, tblOutput);
            g1.setGoogleScrapper(this);
            g1.start();
        }
    }//GEN-LAST:event_btnSearchActionPerformed

    private void btnDownloadCSVFileActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_btnDownloadCSVFileActionPerformed
        FileWriter fw;
        try {
            if(this.getOutputList().isEmpty()){
                logIt("There is no data to save");
                return;
            }
            
            JFileChooser fileChooser = new JFileChooser();
            fileChooser.setDialogTitle("Specify a file to save");
            int userSelection = fileChooser.showSaveDialog(this);
            if (userSelection == JFileChooser.APPROVE_OPTION) {
                File fileToSave = fileChooser.getSelectedFile();
                fw = new FileWriter(fileToSave);
                PrintWriter out = new PrintWriter(fw);
                
                out.print("ROOT_DOMAIN");
                out.print(",");
                out.print("ACTUAL_DESTINATION");
                out.print(",");
                out.print("GOOGLE_ADVERTISING_FLAG");
                out.print(",");
                out.println("KEYWORD");

                for (Output output : outputList) {
                    out.print(output.getRootDomain());
                    out.print(",");
                    out.print(output.getActualDestination());
                    out.print(",");
                    out.print(output.getAdFlag());
                    out.print(",");
                    out.println(output.getKeyword());                    
                }
                out.flush();
                out.close();
                fw.close();
            }
        } catch (IOException ex) {
            Logger.getLogger(GoogleScrapper.class.getName()).log(Level.SEVERE, null, ex);
        }

    }//GEN-LAST:event_btnDownloadCSVFileActionPerformed

    private void rdHttpProxyActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_rdHttpProxyActionPerformed
        // TODO add your handling code here:
    }//GEN-LAST:event_rdHttpProxyActionPerformed

    private void rdMTAllActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_rdMTAllActionPerformed
        // TODO add your handling code here:
    }//GEN-LAST:event_rdMTAllActionPerformed
        private JTextField filename = new JTextField(), dir = new JTextField();
    private boolean selectSearchTermFile() {        
        JFileChooser fc = new JFileChooser();
        File lastOpenedFile = new File(txtKeywordFile.getText());
        if (lastOpenedFile.exists())
        {
            fc.setCurrentDirectory(lastOpenedFile);
        }

        fc.setMultiSelectionEnabled(false);
        int returnVal = fc.showOpenDialog(this);
        if (returnVal == JFileChooser.APPROVE_OPTION)
        {
            File file = fc.getSelectedFile();
            txtKeywordFile.setText(file.getPath());
            logIt(file.getPath() + " file selected!\n");
            readKeywordFile(file);
            return true;
        }
        return false;
    }
    
    private void readKeywordFile(File file){
        try {
            BufferedReader f = new BufferedReader(new FileReader(file));
            String line;
            keyword = new ArrayList<String>();
            while ((line = f.readLine()) != null) {
                keyword.add(line);
                logIt(line + " ");
            }            
            txtKeyword.setText(String.valueOf(keyword.size()));
            logIt(keyword.size() + " keyword found!\n");
        } catch (IOException e) {
            logIt("Error reading keyword file: " + e.getMessage()+ "\n");
        }
    }
    
    private void loadProxy(String url) {
        try {
            Document doc = Jsoup.connect(url).get();
            Element table = doc.getElementById("proxylisttable");
            Elements tbodys = table.getElementsByTag("tbody");
            //Todo clean the code
            Object[][] r = null;int i = 0;
            for (Element tbody : tbodys) {
                Elements trs = tbody.getElementsByTag("tr");
                r = new Object[trs.size()][4];
                i = 0;
                for (Element tr : trs) {
                    Elements tds = tr.getElementsByTag("td");
                    if (tds.get(6).text().equalsIgnoreCase("yes")) {
                        r[i][0] = tds.get(0).text();
                        r[i][1] = tds.get(1).text();
                        r[i][2] = tds.get(5).text();
                        r[i][3] = tds.get(6).text();
                        i = i + 1;
                    }
                }
            }
            if (r != null) {
                rows = new Object[i][4];
                for (int j = 0; j < i; j++) {
                    rows[j][0] = r[j][0];
                    rows[j][1] = r[j][1];
                    rows[j][2] = r[j][2];
                    rows[j][3] = r[j][3];
                }
            }
            
            tblProxyList.setModel(new javax.swing.table.DefaultTableModel(rows, IConstant.headers));
            lblTotal.setText(String.valueOf(rows.length));
        } catch (Throwable t) {
            System.out.println(t);
        }
    }
    
    /**
     * @param args the command line arguments
     */
    public static void main(String args[]) {
        /* Set the Nimbus look and feel */
        //<editor-fold defaultstate="collapsed" desc=" Look and feel setting code (optional) ">
        /* If Nimbus (introduced in Java SE 6) is not available, stay with the default look and feel.
         * For details see http://download.oracle.com/javase/tutorial/uiswing/lookandfeel/plaf.html 
         */
        try {
            for (javax.swing.UIManager.LookAndFeelInfo info : javax.swing.UIManager.getInstalledLookAndFeels()) {
                if ("Nimbus".equals(info.getName())) {
                    javax.swing.UIManager.setLookAndFeel(info.getClassName());
                    break;
                }
            }            
        } catch (ClassNotFoundException ex) {
            java.util.logging.Logger.getLogger(GoogleScrapper.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        } catch (InstantiationException ex) {
            java.util.logging.Logger.getLogger(GoogleScrapper.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        } catch (IllegalAccessException ex) {
            java.util.logging.Logger.getLogger(GoogleScrapper.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        } catch (javax.swing.UnsupportedLookAndFeelException ex) {
            java.util.logging.Logger.getLogger(GoogleScrapper.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
        }
        //</editor-fold>

        /* Create and display the form */
        java.awt.EventQueue.invokeLater(new Runnable() {
            public void run() {
                new GoogleScrapper().setVisible(true);                
            }
        });
    }

    // Variables declaration - do not modify//GEN-BEGIN:variables
    private javax.swing.JButton btnDownloadCSVFile;
    private javax.swing.JButton btnKeywordFile;
    private javax.swing.JButton btnProxyLoad;
    private javax.swing.JButton btnSearch;
    private javax.swing.ButtonGroup buttonGroup1;
    private javax.swing.ButtonGroup buttonGroup2;
    private javax.swing.JTextPane consle;
    private javax.swing.JLabel jLabel1;
    private javax.swing.JLabel jLabel2;
    private javax.swing.JLabel jLabel3;
    private javax.swing.JLabel jLabel4;
    private javax.swing.JLabel jLabel5;
    private javax.swing.JPanel jPanel1;
    private javax.swing.JPanel jPanel2;
    private javax.swing.JScrollPane jScrollPane1;
    private javax.swing.JScrollPane jScrollPane2;
    private javax.swing.JScrollPane jScrollPane3;
    private javax.swing.JTabbedPane jTabbedPane2;
    private javax.swing.JLabel lblSearchTerms;
    private javax.swing.JLabel lblSearchTerms1;
    private javax.swing.JLabel lblTotal;
    private javax.swing.JRadioButton rdHttpProxy;
    private javax.swing.JRadioButton rdHttpsProxy;
    private javax.swing.JRadioButton rdMTAll;
    private javax.swing.JRadioButton rdMTOnlyGooogle;
    private javax.swing.JTable tblOutput;
    private javax.swing.JTable tblProxyList;
    private javax.swing.JTextField txtKeyword;
    private javax.swing.JTextField txtKeywordFile;
    // End of variables declaration//GEN-END:variables

    public StyledDocument doc;
    public StyledDocument getDoc() {
        if(doc == null){
            doc = consle.getStyledDocument();
        }
        return doc;
    }

    public void setDoc(StyledDocument doc) {
        this.doc = doc;
    }

    public Object[][] getRows() {
        return rows;
    }

    public void setRows(Object[][] rows) {
        this.rows = rows;
    }

    public List<String> getKeyword() {
        return keyword;
    }

    public void setKeyword(List<String> keyword) {
        this.keyword = keyword;
    }

    public List<Output> getOutputList() {
        if(outputList == null){
            outputList = new ArrayList<Output>();
        }
        return outputList;
    }

    public void setOutputList(List<Output> outputList) {
        this.outputList = outputList;
    }

    public String getProxy() {
        return proxy;
    }

    public void setProxy(String proxy) {
        this.proxy = proxy;
    }

    public String getThreadSettings() {
        return threadSettings;
    }

    public void setThreadSettings(String threadSettings) {
        this.threadSettings = threadSettings;
    }
    
    @Override
    public void logIt(final String message) {                
        try {
            Runnable pickHighBetaStock = new Runnable() {
                @Override
                public void run() {
                    try {
                        String prev = consle.getText();
                        if(prev.length()>1000){
                            consle.setText("");
                        }
                        getDoc().insertString(getDoc().getLength(), message, null);
                        consle.update(consle.getGraphics());
                    } catch (BadLocationException exc) {
                        System.out.println("Error: " + exc.getMessage());
                    }
                }
            };
            SwingUtilities.invokeLater(pickHighBetaStock);
        } catch (Throwable ex) {
            Logger.getLogger(GoogleScrapper.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    
    @Deprecated
    @Override
    public void updateOutputTables() {
        int i = 0;
        Object[][] outputRows = new Object[outputList.size()][4];
        for(Output output : outputList){
            outputRows[i][0] = output.getRootDomain();
            outputRows[i][1] = output.getActualDestination();
            outputRows[i][2] = output.getAdFlag();
            outputRows[i][3] = output.getKeyword();
            i = i + 1;
        }
        tblOutput.setModel(new javax.swing.table.DefaultTableModel(outputRows, IConstant.outputHeaders));
        tblOutput.update(tblOutput.getGraphics());
    }

    @Override
    public synchronized void addToOutputList(final Output output) {
        try {
            Runnable pickHighBetaStock = new Runnable() {
                @Override
                public void run() {
                    try {
                        getOutputList().add(output);
                        Object[][] outputRows = new Object[1][4];
                        outputRows[0][0] = output.getRootDomain();
                        outputRows[0][1] = output.getActualDestination();
                        outputRows[0][2] = output.getAdFlag();
                        outputRows[0][3] = output.getKeyword();
                        model.addRow(outputRows[0]);
                    } catch (Exception exc) {
                        System.out.println("Error: " + exc.getMessage());
                    }
                }
            };
            SwingUtilities.invokeLater(pickHighBetaStock);
        } catch (Throwable ex) {
            Logger.getLogger(GoogleScrapper.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
}
